#!/usr/bin/env python3
#coding=utf-8

import threading,os,urllib,sys,time
import urllib.request
import urllib.error
import json
import socket
import http.client
import ImgDiskManager
from ProxyData import *

headers = {
    'User-Agent':'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0)'
}

class threadPool(threading.Thread):
    def __init__(self, n):
        threading.Thread.__init__(self)
        self.maxNum = n
        self.queue = []
        self.live = True
        self.startNum = 0
        self.taskList = []
        self.queueLock = threading.Lock()

    def add_task(self, func, args):
        self.queueLock.acquire()
        self.queue.append([func, args])
        self.queueLock.release()

    def close(self):
        self.live = False

    def run(self):
        print('start')
        while self.live:
            time.sleep(0.01)
            if self.queue:
                startNum = 0
                for t in self.taskList:
                    if t and t.isAlive():
                        startNum += 1
                if startNum < self.maxNum:
                    self.queueLock.acquire()
                    task = self.queue.pop(0)
                    self.queueLock.release()
                    taskTh = threading.Thread(target=task[0], args=task[1])
                    taskTh.setDaemon(True)
                    taskTh.start()
                    self.taskList.append(taskTh)
            else:
                startNum = 0
                for t in self.taskList:
                    if t and t.isAlive():
                        startNum += 1
                if not startNum:
                    self.live = False

def downImg(sumInfo, t_lock, imgInfo, useProxy):
    url = imgInfo['url']
    filename = imgInfo['filename']
    request = urllib.request.Request(url, headers=headers)
    if useProxy and OPEN_PROXY:
        request.set_proxy(host=PROXY_HOST, type='http')
    try:
        req = urllib.request.urlopen(request, timeout=8)
        data = req.read()
        req.close()
        t_uri = imgDiskManager.getCanSaveFilePath(filename)
        with open(t_uri, "wb") as f:
           f.write(data)
        ex = ''
    except urllib.error.URLError as e:
        ex = e
    except socket.timeout as e:
        ex = e
    except http.client.IncompleteRead as e:
        ex = e
    except Exception as e:
        ex = e

    t_lock.acquire()
    sumInfo[1] += 1
    t_lock.release()
    print('%s | %d / %d | %s' % (sumInfo[0], sumInfo[1], sumInfo[2], url), ex)

if __name__ == '__main__':
    try:
        with open('conf.json', 'r') as f:
            configJson = f.read()
    except:
        with open('conf.json', 'w') as f:
            f.write('')
        configJson = ''
    try:
        CONFIG_JSON = json.JSONDecoder().decode(configJson)
    except:
        CONFIG_JSON = {}

    if not os.path.exists("./downImg/"):
       os.mkdir("./downImg/")
    imgDiskManager = ImgDiskManager.ImgDiskManager()
    keys = []
    for k, page in CONFIG_JSON.items():
        if not k.startswith('cbcb084'):
            keys.append(k)
    # keys.sort()
    # n = 0
    # for k in keys:
    #     print('%d. %s' % (n, k))
    #     n += 1
    # i = input('请选择： ')
    # useProxy = CONFIG_JSON[keys[int(i)]]['useProxy']
    # downJson = CONFIG_JSON[keys[int(i)]]['pageList']
    # pool = threadPool(4)
    # for sub_page in downJson:
    #     try:
    #         isDelete = sub_page['delete']
    #     except:
    #         isDelete = False
    #     if not isDelete:
    #         for sub_sub_page in sub_page['subPageList']:
    #             try:
    #                 img_list = sub_sub_page['imgList']
    #                 for img_info in img_list:
    #                     pool.add_task(downImg, (img_info, useProxy))
    #             except:
    #                 continue
#
    # pool.setDaemon(True)
    # pool.start()
    # pool.join()
    # keys = ['8kmm/性感美女', '8kmm/丝袜美女', '8kmm/美女模特', '8kmm/美女明星', '8kmm/清纯美女']

    # keys.sort()
    # n = 0
    # for k in keys:
    #     print('%d. %s' % (n, k))
    #     n += 1
    # i = int(input('请选择： '))
    # keys = [keys[i], ]
    # keys = ['nnn2015/欧美图区', ]
    # keys = ['i-ab/纯洁图片', ]
    kLen = len(keys)
    n = 0
    sumInfo = ['', 0, 0]
    for k in keys:
        n += 1
        useProxy = CONFIG_JSON[k]['useProxy']
        downJson = CONFIG_JSON[k]['pageList']
        pool = threadPool(5)
        t_queue = []
        pageNum = 0
        for sub_page in downJson:
            try:
                isDelete = sub_page['delete']
            except:
                isDelete = False
            if not isDelete:
                n = 0
                pageNum += 1
                for sub_sub_page in sub_page['subPageList']:
                    try:
                        img_list = sub_sub_page['imgList']
                        for img_info in img_list:
                            if (not imgDiskManager.fileExists(img_info['filename']))\
                                    or (imgDiskManager.getFileSize(img_info['filename']) < 10240):
                                n += 1
                                t_queue.append(img_info)
                                # if n > 5:
                                #     break
                        # if n > 5:
                        #     break
                    except:
                        continue
            if len(t_queue) > 500 or pageNum > 20:
                break
        sumInfo[0] = k
        sumInfo[2] = len(t_queue)
        t_lock = threading.Lock()
        for img_info in t_queue:
            pool.add_task(downImg, (sumInfo, t_lock, img_info, useProxy))

        pool.setDaemon(True)
        pool.start()
        pool.join()
        sumInfo = ['', 0, 0]
        print(k + ' down Over')
        pass
